
import time

time_start_notebook = time.time()


%%capture
import sys
ENV_COLAB = 'google.colab' in sys.modules

if ENV_COLAB:
    # usual imports
    !pip install watermark
    !pip install scikit-plot

    # Special
    !pip install featuretools[complete]
    !pip install composeml
    !pip install evalml

    print('Environment: Google Colab')


import numpy as np
import pandas as pd
import seaborn as sns
import os,sys,time
import matplotlib.pyplot as plt
import joblib
from tqdm import tqdm_notebook as tqdm
import plotly_express as px
from sklearn.preprocessing import OneHotEncoder
import sklearn.metrics as skmetrics

# special
import imblearn

# featuretools
import composeml as cp
import featuretools as ft
import evalml

# warnings
import warnings

# settings
sns.set()
SEED = 100
pd.set_option('max_columns',100)
pd.set_option('max_colwidth',200)
pd.set_option('plotting.backend','matplotlib') # matplotlib, bokeh, altair, plotly

%matplotlib inline
%load_ext watermark
%watermark -iv

2020-12-21 17:44:22,768 featuretools - WARNING    Featuretools failed to load plugin nlp_primitives from library nlp_primitives. For a full stack trace, set logging to debug.
seaborn       : 0.11.0
pandas        : 1.1.5
plotly_express: 0.4.1
matplotlib    : 3.3.3
imblearn      : 0.7.0
composeml     : 0.5.1
numpy         : 1.19.4
json          : 2.0.9
autopep8      : 1.5.4
sklearn       : 0.23.2
joblib        : 1.0.0
featuretools  : 0.22.0
evalml        : 0.16.1
sys           : 3.8.5 (default, Sep  4 2020, 02:22:02) 
[Clang 10.0.0 ]


def show_methods(obj, ncols=4,contains=None):
    lst = [i for i in dir(obj) if i[0]!='_' ]
    if contains is not None:
        lst = [i for i in lst if contains in i]
    df = pd.DataFrame(np.array_split(lst,ncols)).T.fillna('')
    return df


path_data_train = '../data/processed/train_cleaned.csv'
path_data_test = '../data/processed/test_cleaned.csv'

if ENV_COLAB:
    path_data_train = 'https://raw.githubusercontent.com/bhishanpdl/Datasets/master/Projects/Telco_Customer_Churn/processed/train_cleaned.csv'
    path_data_test = 'https://raw.githubusercontent.com/bhishanpdl/Datasets/master/Projects/Telco_Customer_Churn/processed/test_cleaned.csv'


df_train = pd.read_csv(path_data_train)
df_test = pd.read_csv(path_data_test)

print(df_train.shape)
print(df_test.shape)
df_train.head(2).append(df_train.tail(2))

(5634, 39)
(1409, 39)


target_name = 'Churn'


df_Xtrain  = df_train.drop(target_name,axis=1)
ser_ytrain = df_train[target_name]

df_Xtest   = df_test.drop(target_name,axis=1)
ser_ytest  = df_test[target_name]

ytrain = np.array(ser_ytrain).flatten()
ytest  = np.array(ser_ytest).flatten()


df_Xtrain.apply(lambda x: pd.Series(x).nunique()).nsmallest(5)

MonthlyCharges_cat    1
Gender                2
SeniorCitizen         2
Partner               2
Dependents            2
dtype: int64


cols_drop = ['MonthlyCharges_cat']
df_Xtrain = df_Xtrain.drop(cols_drop,axis=1)
df_Xtest = df_Xtest.drop(cols_drop,axis=1)


# sanity check
print(df_Xtrain.isna().sum().sum(),
      df_Xtest.isna().sum().sum(), 
      df_Xtrain.sum().sum(),
      df_Xtest.sum().sum(),
      ser_ytrain.isna().sum(),
      ser_ytest.isna().sum()
     )

0 0 26646679.3 6773724.300000001 0 0


import plotly_express as px


px.histogram(df_train, x=target_name,height=300,width=300)


px.histogram(df_train, x='Gender', color=target_name,width=300,height=200)


import evalml
from evalml.objectives import CostBenefitMatrix
from evalml import AutoMLSearch


show_methods(evalml)


show_methods(evalml.objectives)


from evalml.objectives import CostBenefitMatrix
cost_benefit_matrix = CostBenefitMatrix(true_positive=400,
                                        true_negative=0,
                                        false_positive=-100,
                                        false_negative=-200)


from evalml import AutoMLSearch


%%time
import warnings
warnings.simplefilter('ignore')


automl = AutoMLSearch(problem_type='binary',
                      objective=cost_benefit_matrix,
                     random_state=SEED,
                    max_time='20 minutes',
                      patience=50,
                      ensembling=True,
                     )


automl.search(df_Xtrain, ser_ytrain)



pipe_cbm = automl.best_pipeline
pipe_cbm.fit(df_Xtrain, ser_ytrain)

`X` passed was not a DataTable. EvalML will try to convert the input as a Woodwork DataTable and types will be inferred. To control this behavior, please pass in a Woodwork DataTable instead.
`y` passed was not a DataColumn. EvalML will try to convert the input as a Woodwork DataTable and types will be inferred. To control this behavior, please pass in a Woodwork DataTable instead.
Generating pipelines to search over...
*****************************
* Beginning pipeline search *
*****************************

Optimizing for Cost Benefit Matrix. 
Greater score is better.

Will stop searching for new pipelines after 1200 seconds.

Allowed model families: linear_model, xgboost, catboost, random_forest, decision_tree, lightgbm, extra_trees

Batch 1: (1/None) Mode Baseline Binary Classification P... Elapsed:00:00
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
Decision Tree Classifier w/ Imputer      Elapsed: 00:00
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 12.247
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 00:01
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 23.536
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
Extra Trees Classifier w/ Imputer        Elapsed: 00:05
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 19.169
High coefficient of variation (cv >= 0.2) within cross validation scores. Extra Trees Classifier w/ Imputer may not perform as estimated on unseen data.
Elastic Net Classifier w/ Imputer + S... Elapsed: 00:07
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
CatBoost Classifier w/ Imputer           Elapsed: 00:08
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 9.780
High coefficient of variation (cv >= 0.2) within cross validation scores. CatBoost Classifier w/ Imputer may not perform as estimated on unseen data.
XGBoost Classifier w/ Imputer            Elapsed: 00:09
	Starting cross validation
[17:55:31] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:55:32] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:55:33] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 22.293
Random Forest Classifier w/ Imputer      Elapsed: 00:12
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 19.649
Logistic Regression Classifier w/ Imp... Elapsed: 00:14
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 26.340
Logistic Regression Classifier w/ Imp... Elapsed: 00:18
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 28.612
Logistic Regression Classifier w/ Imp... Elapsed: 00:20
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.991
Logistic Regression Classifier w/ Imp... Elapsed: 00:22
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 28.168
High coefficient of variation (cv >= 0.2) within cross validation scores. Logistic Regression Classifier w/ Imputer + Standard Scaler may not perform as estimated on unseen data.
Logistic Regression Classifier w/ Imp... Elapsed: 00:24
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.689
Logistic Regression Classifier w/ Imp... Elapsed: 00:26
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.955
LightGBM Classifier w/ Imputer           Elapsed: 00:28
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 28.044
LightGBM Classifier w/ Imputer           Elapsed: 00:29
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 18.087
LightGBM Classifier w/ Imputer           Elapsed: 00:31
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 11.200
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 00:33
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 18.974
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 00:34
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 26.074
XGBoost Classifier w/ Imputer            Elapsed: 00:36
	Starting cross validation
[17:55:57] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:00] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:02] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 17.483
XGBoost Classifier w/ Imputer            Elapsed: 00:45
	Starting cross validation
[17:56:06] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:08] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:11] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 16.400
XGBoost Classifier w/ Imputer            Elapsed: 00:52
	Starting cross validation
[17:56:13] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:19] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:26] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 19.453
XGBoost Classifier w/ Imputer            Elapsed: 01:11
	Starting cross validation
[17:56:32] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:34] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:37] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 16.507
High coefficient of variation (cv >= 0.2) within cross validation scores. XGBoost Classifier w/ Imputer may not perform as estimated on unseen data.
XGBoost Classifier w/ Imputer            Elapsed: 01:18
	Starting cross validation
[17:56:39] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:42] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:56:45] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 18.548
Random Forest Classifier w/ Imputer      Elapsed: 01:27
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 20.678
High coefficient of variation (cv >= 0.2) within cross validation scores. Random Forest Classifier w/ Imputer may not perform as estimated on unseen data.
Random Forest Classifier w/ Imputer      Elapsed: 01:35
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 23.695
Random Forest Classifier w/ Imputer      Elapsed: 01:42
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 23.553
Random Forest Classifier w/ Imputer      Elapsed: 01:47
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 21.494
Random Forest Classifier w/ Imputer      Elapsed: 01:50
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 11.377
High coefficient of variation (cv >= 0.2) within cross validation scores. Random Forest Classifier w/ Imputer may not perform as estimated on unseen data.
Extra Trees Classifier w/ Imputer        Elapsed: 01:53
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 22.080
High coefficient of variation (cv >= 0.2) within cross validation scores. Extra Trees Classifier w/ Imputer may not perform as estimated on unseen data.
Extra Trees Classifier w/ Imputer        Elapsed: 01:57
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 23.269
Extra Trees Classifier w/ Imputer        Elapsed: 01:58
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 10.561
High coefficient of variation (cv >= 0.2) within cross validation scores. Extra Trees Classifier w/ Imputer may not perform as estimated on unseen data.
Extra Trees Classifier w/ Imputer        Elapsed: 02:02
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 7.561
High coefficient of variation (cv >= 0.2) within cross validation scores. Extra Trees Classifier w/ Imputer may not perform as estimated on unseen data.
Extra Trees Classifier w/ Imputer        Elapsed: 02:05
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 18.779
High coefficient of variation (cv >= 0.2) within cross validation scores. Extra Trees Classifier w/ Imputer may not perform as estimated on unseen data.
Decision Tree Classifier w/ Imputer      Elapsed: 02:10
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 13.720
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
Decision Tree Classifier w/ Imputer      Elapsed: 02:11
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 20.323
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
Decision Tree Classifier w/ Imputer      Elapsed: 02:12
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -1.846
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
Decision Tree Classifier w/ Imputer      Elapsed: 02:13
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 15.264
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
Decision Tree Classifier w/ Imputer      Elapsed: 02:14
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 14.679
High coefficient of variation (cv >= 0.2) within cross validation scores. Decision Tree Classifier w/ Imputer may not perform as estimated on unseen data.
CatBoost Classifier w/ Imputer           Elapsed: 02:16
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 17.448
CatBoost Classifier w/ Imputer           Elapsed: 02:18
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 20.944
High coefficient of variation (cv >= 0.2) within cross validation scores. CatBoost Classifier w/ Imputer may not perform as estimated on unseen data.
CatBoost Classifier w/ Imputer           Elapsed: 02:20
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 20.714
CatBoost Classifier w/ Imputer           Elapsed: 02:21
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 12.123
CatBoost Classifier w/ Imputer           Elapsed: 02:23
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 22.453
Elastic Net Classifier w/ Imputer + S... Elapsed: 02:25
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
Elastic Net Classifier w/ Imputer + S... Elapsed: 02:26
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
Elastic Net Classifier w/ Imputer + S... Elapsed: 02:27
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
Elastic Net Classifier w/ Imputer + S... Elapsed: 02:29
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -32.836
Elastic Net Classifier w/ Imputer + S... Elapsed: 02:30
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: -53.071
Stacked Ensemble Classification Pipeline Elapsed: 02:31
	Starting cross validation
[17:57:54] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:58:02] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:58:03] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
[17:58:03] WARNING: /Users/runner/miniforge3/conda-bld/xgboost_1607604592557/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.
	Finished cross validation - mean Cost Benefit Matrix: 25.719
Logistic Regression Classifier w/ Imp... Elapsed: 02:49
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 26.251
High coefficient of variation (cv >= 0.2) within cross validation scores. Logistic Regression Classifier w/ Imputer + Standard Scaler may not perform as estimated on unseen data.
Logistic Regression Classifier w/ Imp... Elapsed: 02:51
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 28.275
Logistic Regression Classifier w/ Imp... Elapsed: 02:53
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.707
Logistic Regression Classifier w/ Imp... Elapsed: 02:55
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.086
Logistic Regression Classifier w/ Imp... Elapsed: 02:57
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 28.346
LightGBM Classifier w/ Imputer           Elapsed: 03:00
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 24.494
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 03:01
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 16.986
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 03:03
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 18.495
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.
LightGBM Classifier w/ Imputer           Elapsed: 03:04
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 24.423
LightGBM Classifier w/ Imputer           Elapsed: 03:06
	Starting cross validation
	Finished cross validation - mean Cost Benefit Matrix: 27.955
High coefficient of variation (cv >= 0.2) within cross validation scores. LightGBM Classifier w/ Imputer may not perform as estimated on unseen data.


50 iterations without improvement. Stopping search early...

Search finished after 03:09            
Best pipeline: Logistic Regression Classifier w/ Imputer + Standard Scaler
Best pipeline Cost Benefit Matrix: 28.611999
CPU times: user 6min 47s, sys: 14 s, total: 7min 1s
Wall time: 3min 9s

GeneratedPipeline(parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'median', 'categorical_fill_value': None, 'numeric_fill_value': None}, 'Logistic Regression Classifier':{'penalty': 'l2', 'C': 6.367449446597125, 'n_jobs': -1, 'multi_class': 'auto', 'solver': 'lbfgs'},})


# automl.search?


pipe_score_cbm = pipe_cbm.score(df_Xtest, ser_ytest, [cost_benefit_matrix])
print (pipe_score_cbm)

# Calculate total profit across all customers using
# pipeline optimized for Log Loss
total_profit_cbm = pipe_score_cbm['Cost Benefit Matrix'] * len(df_Xtrain)
print (f"{total_profit_cbm:,.2f}")

OrderedDict([('Cost Benefit Matrix', 24.059616749467715)])
135,551.88


pipe_cbm

GeneratedPipeline(parameters={'Imputer':{'categorical_impute_strategy': 'most_frequent', 'numeric_impute_strategy': 'median', 'categorical_fill_value': None, 'numeric_fill_value': None}, 'Logistic Regression Classifier':{'penalty': 'l2', 'C': 6.367449446597125, 'n_jobs': -1, 'multi_class': 'auto', 'solver': 'lbfgs'},})


show_methods(evalml.model_understanding)


from evalml.model_understanding import graphs as evgraphs
show_methods(evgraphs)


show_methods(pipe_cbm)


ytest = np.array(ser_ytest).flatten()
ypreds = np.array(pipe_cbm.predict(df_Xtest)).flatten()
yprobs2d = pipe_cbm.predict_proba(df_Xtest).to_numpy()
yprobs = yprobs2d[:,1]


evgraphs.graph_confusion_matrix(ytest, ypreds)


evgraphs.graph_roc_curve(ytest,yprobs)


evgraphs.graph_precision_recall_curve(ytest,yprobs)


show_methods(evgraphs,contains='graph')


evgraphs.graph_partial_dependence(pipe_cbm,df_Xtest,feature='SeniorCitizen')


N = 100
evgraphs.graph_binary_objective_vs_threshold(
    pipe_cbm,df_Xtest.iloc[:N],ytest[:N],objective=cost_benefit_matrix)


%%time
evgraphs.graph_permutation_importance(pipe_cbm,df_Xtest.iloc[:N],ytest[:N],
                                      objective=cost_benefit_matrix)


import sklearn.metrics as skmetrics
import scikitplot.metrics as skpmetrics


def model_eval_bin(model_name,ytest,ypreds,yprobs2d,show_plots=True):
    import sklearn.metrics as skmetrics
    import scikitplot.metrics as skpmetrics
    import os

    acc       = skmetrics.accuracy_score(ytest,ypreds)
    precision = skmetrics.precision_score(ytest,ypreds)
    recall    = skmetrics.recall_score(ytest,ypreds)
    f1        = skmetrics.f1_score(ytest,ypreds)
    auc       = skmetrics.roc_auc_score(ytest,ypreds)

    print(skmetrics.classification_report(ytest,ypreds))
    print(skmetrics.confusion_matrix(ytest,ypreds))

    df_res = pd.DataFrame({'Accuracy':[acc],
                          'Precision': [precision],
                          'Recall': [recall],
                          'F1-score': [f1],
                          'AUC': [auc]},index=[model_name])

    display(df_res.style.format("{:.4f}"))
    if not os.path.isdir('../outputs'):
        os.makedirs('../outputs')
    o = '.' if ENV_COLAB else '../outputs/'
    df_res.to_csv(o+f'model_{model_name}.csv',index=True)

    if show_plots:
        skpmetrics.plot_precision_recall(ytest,yprobs2d) # more focus on minority
        skpmetrics.plot_roc_curve(ytest,yprobs2d) # equal focus on both groups
        skpmetrics.plot_confusion_matrix(ytest,ypreds)

model_eval_bin('evalml',ytest,ypreds,yprobs2d,show_plots=True)

              precision    recall  f1-score   support

           0       0.84      0.89      0.86      1035
           1       0.63      0.54      0.58       374

    accuracy                           0.79      1409
   macro avg       0.73      0.71      0.72      1409
weighted avg       0.78      0.79      0.79      1409

[[916 119]
 [173 201]]


time_taken = time.time() - time_start_notebook
h,m = divmod(time_taken,60*60)
print('Time taken to run whole notebook: {:.0f} hr '\
      '{:.0f} min {:.0f} secs'.format(h, *divmod(m,60)))

Time taken to run whole notebook: 0 hr 24 min 16 secs

	Gender	Partner	Dependents	Tenure	PhoneService	MultipleLines	InternetService	OnlineSecurity	OnlineBackup	DeviceProtection	TechSupport	StreamingTV	StreamingMovies	Contract	PaperlessBilling	PaymentMethod	MonthlyCharges	TotalCharges	Contract_Month-to-month	NoSeniorCitizen_Contract_Month-to-month	InternetService_Fiber optic	StreamingTV_NoInternetService	No_OB_DP_TS	TotalServices	SenCit_Dependents	Partner_Dependents	SenCit_Partner	SenCit_Contract	SenCit_TechSupport	SenCit_PayMeth	Contract_mean_totCharges	Contract_totCharges_diff	PayMeth_mean_monthCharges	PayMeth_monthCharges_diff	Tenure_cat
0	0	1	0	36	1	2	1	2	2	2	2	0	2	2	1	1	106.05	3834.40	1	0	0	1	1	2	0	1	1	2	2	1	3683.643192	150.756808	66.703657	39.346343	3
1	1	0	0	10	1	0	0	2	0	0	2	2	0	0	0	1	62.25	612.95	0	1	1	1	1	1	0	0	0	0	2	1	1370.923131	-757.973131	66.703657	-4.453657	0
5632	0	1	1	68	1	2	1	0	2	0	2	2	2	2	1	1	103.75	7039.45	1	0	0	1	1	2	1	2	1	2	2	1	3683.643192	3355.806808	66.703657	37.046343	5
5633	1	0	0	69	1	2	2	1	1	1	1	1	1	2	0	1	23.95	1713.10	1	0	1	0	1	7	0	0	0	2	1	1	3683.643192	-1970.543192	66.703657	-42.753657	5

	0	1	2	3
0	AUC	F1Weighted	Precision	cost_benefit_matrix
1	AUCMacro	FraudCost	PrecisionMacro	fraud_cost
2	AUCMicro	LeadScoring	PrecisionMicro	get_all_objective_names
3	AUCWeighted	LogLossBinary	PrecisionWeighted	get_core_objective_names
4	AccuracyBinary	LogLossMulticlass	R2	get_core_objectives
5	AccuracyMulticlass	MAE	Recall	get_non_core_objectives
6	BalancedAccuracyBinary	MCCBinary	RecallMacro	get_objective
7	BalancedAccuracyMulticlass	MCCMulticlass	RecallMicro	lead_scoring
8	BinaryClassificationObjective	MSE	RecallWeighted	multiclass_classification_objective
9	CostBenefitMatrix	MaxError	RegressionObjective	objective_base
10	ExpVariance	MeanSquaredLogError	RootMeanSquaredError	regression_objective
11	F1	MedianAE	RootMeanSquaredLogError	standard_metrics
12	F1Macro	MulticlassClassificationObjective	binary_classification_objective	utils
13	F1Micro	ObjectiveBase

	0	1	2	3
0	LabelBinarizer	get_objective	import_or_raise	sk_partial_dependence
1	ModelFamily	graph_binary_objective_vs_threshold	jupyter_check	sk_permutation_importance
2	NullsInColumnWarning	graph_confusion_matrix	normalize_confusion_matrix	sklearn_auc
3	ProblemTypes	graph_partial_dependence	np	sklearn_confusion_matrix
4	binary_objective_vs_threshold	graph_permutation_importance	partial_dependence	sklearn_precision_recall_curve
5	calculate_permutation_importance	graph_precision_recall_curve	pd	sklearn_roc_curve
6	confusion_matrix	graph_prediction_vs_actual	precision_recall_curve	unique_labels
7	copy	graph_roc_curve	roc_curve	warnings
8	evalml

	0	1	2	3
0	classes_	describe	hyperparameters	predict_proba
1	clone	estimator	input_feature_names	problem_type
2	component_graph	feature_importance	load	random_state
3	compute_estimator_features	fit	model_family	save
4	custom_hyperparameters	get_component	name	score
5	custom_name	graph	parameters	summary
6	default_parameters	graph_feature_importance	predict	threshold

Modelling Customer Churn

Load the libraries

Colab¶

Useful Scripts

Load the Data

Data Processing

Train and Test Data¶

Sanity Checks¶

Class Balance¶

Modelling: evalML

Select the Objective

AutoML Search

Ojbective: Cost Benefit Matrix

Model Evaluation

Time Taken

	0	1	2	3
0	AutoMLSearch	exceptions	preprocessing	skopt
1	automl	model_family	print_info	tuners
2	data_checks	model_understanding	problem_types	utils
3	demos	objectives	sklearn	warnings
4	evalml	pipelines

	0	1	2	3
0	binary_objective_vs_threshold	explain_predictions_best_worst	graph_precision_recall_curve	partial_dependence
1	calculate_permutation_importance	graph_binary_objective_vs_threshold	graph_prediction_vs_actual	precision_recall_curve
2	confusion_matrix	graph_confusion_matrix	graph_roc_curve	prediction_explanations
3	explain_prediction	graph_partial_dependence	graphs	roc_curve
4	explain_predictions	graph_permutation_importance	normalize_confusion_matrix